package de.preiscrawler.parser.amazon;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;

import net.htmlparser.jericho.Element;
import net.htmlparser.jericho.Source;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.methods.GetMethod;

import de.preisrobot.service.helper.HttpClientFactory;

public class AmazonCategorieCrawler {

	private static String amazon_url = "http://www.amazon.de";

	public static void main(String[] args) throws HttpException, IOException {

		String sourceUrlString = "http://www.amazon.de/b/ref=sa_menu_desk3?ie=UTF8&node=514699031";// "http://www.idealo.de/";

		List<String> links = getCategorieLinks(sourceUrlString);
		List<String> links2 = new ArrayList<String>();
		
		for (String string : links) {
			List<String> list = getCategorieLinks(sourceUrlString);
			links2.addAll(list);
		}
		
		
		System.out.println("size "+links2.size());
		
		
		FileOutputStream out = new FileOutputStream("d:\\amazonlinks.txt");
		
		for (String string : links2) {
			out.write(string.getBytes());
			
		}
		out.flush();
		out.close();
		
		
		
		
	}

	private static List<String> getCategorieLinks(String sourceUrlString)
			throws IOException, HttpException {
		HttpClient client = HttpClientFactory.getHttpClient();

		GetMethod method = HttpClientFactory.createGETMethod(sourceUrlString);
		client.executeMethod(method);

		try {

			InputStream in = method.getResponseBodyAsStream();
			return parseStream(in);

		} catch (Exception e) {
			e.printStackTrace();

		} finally {

			method.releaseConnection();
		}
		return null;
	}

	private static List<String> parseStream(InputStream in) throws Exception {
		 
		List<String> catUrls = new ArrayList<String>();
		
		Source source = new Source(in);

		List<Element> allElementsByClass = source
				.getAllElementsByClass("left_nav");

		if(allElementsByClass == null || allElementsByClass.isEmpty())
			return catUrls;
		
		List<Element> elementList = allElementsByClass.get(0).getAllElements(
				"a");

		for (Element element : elementList) {
			String attributeValue = element.getAttributeValue("href");

			if ( ! attributeValue.startsWith(amazon_url)) {
				attributeValue = amazon_url + attributeValue;

				System.out.println(attributeValue);
				
				catUrls.add(attributeValue);
			}
		}
			return catUrls;
	}
}
